# # encoding=utf-8
# import random
# import uuid
#
# from zc_core.dao.cookie_dao import CookieDao
#
# from ctaxccgp.utils.area_helper import AreaHelper
# from zc_core.middlewares.agents.user_agents import agents
# from zc_core.util.encrypt_util import md5
#
#
# class ZcySkuMiddleware(object):
#     """
#     商品列表页/供应商列表页
#     专用反爬中间件
#     """
#     def __init__(self):
#         self.referer_tpl = [
#             'https://ctaxccgp.zcygov.cn/',
#             'https://ctaxccgp.zcygov.cn/eevees/shop?shopId={}&searchType=1&utm=a0017.2ef5001f.cl47.2.{}',
#         ]
#         self.area_helper = AreaHelper()
#         # proxy -> log_client_uuid
#         self.log_client_map = dict()
#         # proxy -> area_id
#         self.area_map = dict()
#
#     def process_request(self, request, spider):
#         meta = request.meta
#         proxy = meta.get('proxy', 'NO_PROXY')
#         req_type = meta.get('reqType', None)
#         if req_type in ['sku', 'supplier']:
#             request.headers['User-Agent'] = random.choice(agents)
#             log_client = self.log_client_map.get(proxy)
#             if not log_client:
#                 log_client = str(uuid.uuid4())
#                 self.log_client_map[proxy] = log_client
#             area_id = self.area_map.get(proxy)
#             if not area_id:
#                 area_id = self.area_helper.random_choice()
#                 self.area_map[proxy] = area_id
#             request.headers['Cookie'] = '_zcy_log_client_uuid={}; aid={}; districtCode=001000'.format(log_client, area_id)
#         if req_type in ['sku']:
#             sp_id = meta.get('supplierId')
#             request.headers['Referer'] = random.choice(self.referer_tpl).format(sp_id, md5(str(uuid.uuid4())))
#
#     def process_response(self, request, response, spider):
#         meta = request.meta
#         proxy = meta.get('proxy', 'NO_PROXY')
#         req_type = meta.get('reqType', None)
#         if req_type in ['sku'] and response.status == 405:
#             # 只观察，不处理
#             spider.logger.info('触发反爬: proxy=%s' % proxy)
#             self.log_client_map.pop(proxy, None)
#
#         return response
#
#     def process_exception(self, request, exception, spider):
#         meta = request.meta
#         proxy = meta.get('proxy', 'NO_PROXY')
#         req_type = meta.get('reqType', None)
#         if req_type in ['sku', 'supplier']:
#             # 只观察，不处理
#             spider.logger.info('反爬异常: proxy=%s, ex=%s' % (proxy, str(type(exception))))
#
#
# class ZcyItemMiddleware(object):
#     # 持续触发反爬而终止采集的次数阈值
#     max_bind_quit = 30
#     # 持续触发反爬次数记录
#     bind_request_count = 0
#     """
#     商品详情页
#     专用反爬中间件
#     """
#     def __init__(self):
#         self.referer_tpl = [
#             'https://ctaxccgp.zcygov.cn/items/{}?searchType=1',
#             'https://ctaxccgp.zcygov.cn/items/{}?utm=utm=a0017.b0231.cl47.10.f7920050b2ef11eaa6c3cf86e55f5c88&skuId={}',
#             'https://ctaxccgp.zcygov.cn/items/{}?searchType=1&utm=a0004.eevees-search.c0002.d0003.{}',
#         ]
#         self.area_helper = AreaHelper()
#         # proxy -> log_client_uuid
#         self.log_client_map = dict()
#         # proxy -> area_id
#         self.area_map = dict()
#
#     def process_request(self, request, spider):
#         meta = request.meta
#         proxy = meta.get('proxy', 'NO_PROXY')
#         sku_id = meta.get('skuId', '')
#         session_id = meta.get('session', '')
#         req_type = meta.get('reqType', None)
#         if sku_id and 'item' == req_type:
#             request.headers['User-Agent'] = random.choice(agents)
#             # request.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:79.0) Gecko/20100101 Firefox/79.0'
#             # request.headers['Referer'] = 'https://ctaxccgp.zcygov.cn/items/{}?searchType=1&utm=221122.3d54c815.0.0.{}'.format(sku_id, str(uuid.uuid4()))
#             request.headers['Referer'] = 'https://ctaxccgp.zcygov.cn/items/{}?searchType=1&utm=a2117.b0963.cl47.11.34fgrtee5453'.format(sku_id)
#             log_client = self.log_client_map.get(proxy)
#             acw_tc = str('{}{}'.format(md5(log_client), log_client))
#             if not log_client:
#                 log_client = str(uuid.uuid4())
#                 self.log_client_map[proxy] = log_client
#             area_id = self.area_map.get(proxy)
#             if not area_id:
#                 area_id = self.area_helper.random_choice()
#                 self.area_map[proxy] = area_id
#             # request.headers['Cookie'] = 'SESSION={}'.format(session_id)
#             # request.headers['Cookie'] = '_zcy_log_client_uuid={}; acw_tc={}; aid={}; districtCode={}; SESSION={}'.format(log_client, acw_tc, area_id, area_id, session_id)
#             request.headers['Cookie'] = '_zcy_log_client_uuid=5543fc70-1471-89eb-4545-5f6f0c64dbc64; acw_tc=e5cdgrd016082035933594453e649508be6b90a916f618c6c315abce4ab7545; aid={}; districtCode={}; SESSION={}'.format(area_id, area_id, session_id)
#             # request.headers['Cookie'] = '_zcy_log_client_uuid=d6g2fc50-3b5b-11eb-adca-138a88c09312; districtCode=001003; districtName=%E5%9B%BD%E5%AE%B6%E7%A8%8E%E5%8A%A1%E6%80%BB%E5%B1%80%E6%9C%AC%E7%BA%A7; aid=310202; acw_tc=66520ff416076551702483626e2swc74fdde3c70972cf9f9cf2b84f0fec987; SESSION={}'.format(session_id)
#
#     def process_response(self, request, response, spider):
#         meta = request.meta
#         proxy = meta.get('proxy', 'NO_PROXY')
#         sku_id = meta.get('skuId', '')
#         req_type = meta.get('reqType', None)
#         if sku_id and 'item' == req_type and response.status == 405:
#             self.bind_request_count = self.bind_request_count + 1
#             # 只观察，不处理
#             spider.logger.info('触发反爬[%s]: proxy=%s, sku=%s' % (self.bind_request_count, proxy, sku_id))
#             self.log_client_map.pop(proxy, None)
#         else:
#             self.bind_request_count = 0
#         if self.bind_request_count > self.max_bind_quit:
#             # 清理cookie
#             cookie_dao = CookieDao()
#             cookie_dao.delete_cookie()
#             # 终止采集
#             spider.crawler.engine.close_spider(spider, '连续触发反爬次数超过阈值终止采集')
#
#         return response
#
#     def process_exception(self, request, exception, spider):
#         meta = request.meta
#         proxy = meta.get('proxy', 'NO_PROXY')
#         sku_id = meta.get('skuId', '')
#         req_type = meta.get('reqType', None)
#         if sku_id and 'item' == req_type:
#             # 只观察，不处理
#             spider.logger.info('反爬异常: proxy=%s, sku=%s, ex=%s' % (proxy, sku_id, str(type(exception))))
